<!DOCTYPE html>
<html lang="en">

<!-- Head tag -->
<head>
    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="google-site-verification" content="xBT4GhYoi5qRD5tr338pgPM5OWHHIDR6mNg1a3euekI" />
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <meta name="description" content="">
    <meta name="keyword"  content="">
    <link rel="shortcut icon" href="/img/ironman-draw.png">
    <!-- Place this tag in your head or just before your close body tag. -->
    <script async defer src="https://buttons.github.io/buttons.js"></script>
    <title>
        
          python爬虫入门实战之爬取美国体育网篮球比赛数据（selenium+xpath） - 赵先生
        
    </title>

    <link rel="canonical" href="http://zqqpluto.xyz/article/python爬虫入门实战之爬取美国体育网篮球比赛数据（selenium+xpath）/">

    <!-- Bootstrap Core CSS -->
    <link rel="stylesheet" href="/css/bootstrap.min.css">

    <!-- Custom CSS --> 
    <link rel="stylesheet" href="/css/beantech.min.css">

    <link rel="stylesheet" href="/css/donate.css">
    
    <!-- Pygments Highlight CSS -->
    <link rel="stylesheet" href="/css/highlight.css">

    <link rel="stylesheet" href="/css/widget.css">

    <link rel="stylesheet" href="/css/rocket.css">

    <link rel="stylesheet" href="/css/signature.css">

    <link rel="stylesheet" href="/css/toc.css">

    <!-- Custom Fonts -->
    <!-- <link href="https://maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css" rel="stylesheet" type="text/css"> -->
    <!-- Hux change font-awesome CDN to qiniu -->
    <link href="https://cdn.staticfile.org/font-awesome/4.5.0/css/font-awesome.min.css" rel="stylesheet" type="text/css">


    <!-- Hux Delete, sad but pending in China
    <link href='http://fonts.googleapis.com/css?family=Lora:400,700,400italic,700italic' rel='stylesheet' type='text/css'>
    <link href='http://fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,600italic,700italic,800italic,400,300,600,700,800' rel='stylesheet' type='text/
    css'>
    -->


    <!-- HTML5 Shim and Respond.js IE8 support of HTML5 elements and media queries -->
    <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
    <!--[if lt IE 9]>
        <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
        <script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script>
    <![endif]-->

    <!-- ga & ba script hoook -->
    <script></script>
</head>


<!-- hack iOS CSS :active style -->
<body ontouchstart="">
	<!-- Modified by Yu-Hsuan Yen -->
<!-- Post Header -->
<style type="text/css">
    header.intro-header {
        background-image: url('/img/article_header/article_header.png')
                /*post*/
                
    }

    
</style>

<header class="intro-header">
    <!-- Signature -->
    <div id="signature">
        <div class="container">
            <div class="row">
                <div class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">
                    
                    <div class="post-heading">
                        <div class="tags">
                            
                            <a class="tag" href="/tags/#python"
                                title="python">python</a>
                            
                            <a class="tag" href="/tags/#xpath"
                                title="xpath">xpath</a>
                            
                            <a class="tag" href="/tags/#selenium"
                                title="selenium">selenium</a>
                            
                            <a class="tag" href="/tags/#爬虫"
                                title="爬虫">爬虫</a>
                            
                        </div>
                        <h1>python爬虫入门实战之爬取美国体育网篮球比赛数据（selenium+xpath）</h1>
                        <h2 class="subheading">python爬虫</h2>
                        <span class="meta"  id="busuanzi_container_page_pv" >
                            Posted by 赵先生 on
                            2019-08-16   
                            &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; 本文总阅读量<span id="busuanzi_value_page_pv"></span>次
                        </span>

                       

                    </div>
                    


                </div>

            </div>
        </div>
    </div>
</header>
	
    <!-- Navigation -->
<nav class="navbar navbar-default navbar-custom navbar-fixed-top">
    <div class="container-fluid">
        <!-- Brand and toggle get grouped for better mobile display -->
        <div class="navbar-header page-scroll">
            <button type="button" class="navbar-toggle">
                <span class="sr-only">Toggle navigation</span>
                <span class="icon-bar"></span>
                <span class="icon-bar"></span>
                <span class="icon-bar"></span>
            </button>
            <a class="navbar-brand" href="/">赵先生</a>
        </div>

        <!-- Collect the nav links, forms, and other content for toggling -->
        <!-- Known Issue, found by Hux:
            <nav>'s height woule be hold on by its content.
            so, when navbar scale out, the <nav> will cover tags.
            also mask any touch event of tags, unfortunately.
        -->
        <div id="huxblog_navbar">
            <div class="navbar-collapse">
                <ul class="nav navbar-nav navbar-right">
                    <li>
                        <a href="/">Home</a>
                    </li>

                    

                        
                    

                        
                        <li>
                            <a href="/archive/">Archives</a>
                        </li>
                        
                    

                        
                        <li>
                            <a href="/about/">About</a>
                        </li>
                        
                    

                        
                        <li>
                            <a href="/tags/">Tags</a>
                        </li>
                        
                    
                    
                </ul>
            </div>
        </div>
        <!-- /.navbar-collapse -->
    </div>
    <!-- /.container -->
</nav>
<script>
    // Drop Bootstarp low-performance Navbar
    // Use customize navbar with high-quality material design animation
    // in high-perf jank-free CSS3 implementation
    var $body   = document.body;
    var $toggle = document.querySelector('.navbar-toggle');
    var $navbar = document.querySelector('#huxblog_navbar');
    var $collapse = document.querySelector('.navbar-collapse');

    $toggle.addEventListener('click', handleMagic)
    function handleMagic(e){
        if ($navbar.className.indexOf('in') > 0) {
        // CLOSE
            $navbar.className = " ";
            // wait until animation end.
            setTimeout(function(){
                // prevent frequently toggle
                if($navbar.className.indexOf('in') < 0) {
                    $collapse.style.height = "0px"
                }
            },400)
        }else{
        // OPEN
            $collapse.style.height = "auto"
            $navbar.className += " in";
        }
    }
</script>


    <!-- Main Content -->
    <!-- Modify by Yu-Hsuan Yen -->

<!-- Post Content -->
<article>
    <div class="container">
        <div class="row">

            <!-- Post Container -->
            <div class="
                col-lg-8 col-lg-offset-2
                col-md-10 col-md-offset-1
                post-container">

                <h2 id="一-观察url及要爬取的数据">一、观察url及要爬取的数据</h2>
<p><img src="//zqqpluto.xyz/article/python爬虫入门实战之爬取美国体育网篮球比赛数据（selenium+xpath）/20190118164515834.png" alt="img"></p>
<blockquote>
<ol>
<li>URL后面跟着a-index.html，我们再点开几个几面，发现它的链接后缀都是“小写字母-index.html”的格式结尾的，随意我们姜后缀链接进行构造拼接，将最终的URL拼接好后存储到一个容器。</li>
<li>接下来我们对数据分析，经过观察后发现所有的数据我们都是需要的，没有不需要的数据。（如果有无用的数，那么我们在爬取数据的时候就不爬取这一项）</li>
<li>在爬取的过程中发现以x-index.html结尾的链接是无法打开的，寻找问题之后发现是没有这个链接，x字体是黑色的。故在URL的构造的时候就不要构造此URL。</li>
</ol>
</blockquote>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 构造链接尾部方法 </span></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">getUrls</span><span class="params">(self,head_url)</span>:</span></span><br><span class="line">        urls = []</span><br><span class="line">        <span class="keyword">for</span> i <span class="keyword">in</span> range(<span class="number">97</span>,<span class="number">123</span>):</span><br><span class="line">            <span class="keyword">if</span> i == <span class="number">120</span>:</span><br><span class="line">                <span class="keyword">pass</span></span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                url = head_url</span><br><span class="line">                url += chr(i)+<span class="string">'-index.html'</span></span><br><span class="line">                urls.append(url)</span><br><span class="line">        <span class="keyword">return</span> urls</span><br></pre></td></tr></table></figure>
<p><img src="//zqqpluto.xyz/article/python爬虫入门实战之爬取美国体育网篮球比赛数据（selenium+xpath）/201901181657438.png" alt="img"></p>
<p>需要注意的是在爬取数据的时候不要把这种无用的数据爬取下来</p>
<h2 id="二-对网页结构进行分析">二、对网页结构进行分析</h2>
<p>按F12查看网页代码，找到要爬取的数据不放呢。不难发现要爬取的数据都在tbody里边。</p>
<p><img src="//zqqpluto.xyz/article/python爬虫入门实战之爬取美国体育网篮球比赛数据（selenium+xpath）/20190118170216406.png" alt="img"></p>
<p>接下来我们查看要爬取的数据在具体哪一个标签内部，找到每一个tr内部的标签，不难发现每一个td都对应了一个数据，不同的是有两个数据是是在td标签下的a标签里面。</p>
<p><img src="//zqqpluto.xyz/article/python爬虫入门实战之爬取美国体育网篮球比赛数据（selenium+xpath）/20190118170749690.png" alt="img"></p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre></td><td class="code"><pre><span class="line">trTags = tree.xpath(<span class="string">'//tr[@data-row and not(@class)]'</span>)<span class="comment">#通过xpath寻找有用的数据，</span></span><br><span class="line">            <span class="keyword">for</span> tag <span class="keyword">in</span> trTags:</span><br><span class="line">                <span class="keyword">try</span>:</span><br><span class="line">                    item = []</span><br><span class="line">                    <span class="comment">#xpath找出的是列表，故要在后面加[0]，不然结果会有[]号</span></span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./th[1]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    <span class="comment"># 这个数据是存在a标签内部的</span></span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[1]/a/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[2]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[3]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[4]/a/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[5]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[6]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[7]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[8]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[9]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[10]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[11]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[12]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[13]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[14]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    items.append(item)</span><br><span class="line">                    item = []</span><br><span class="line">                <span class="keyword">except</span>:</span><br><span class="line">                    <span class="keyword">pass</span></span><br></pre></td></tr></table></figure>
<h2 id="三-数据爬取">三、数据爬取</h2>
<p>在爬取数据的时候我开始使用的是Requests进行请求响应，但是之后我解析的时候发现得到的代码里面没有我需要的数据，这个问题我查询是因为这种网页是js动态导入的，对于这种我的解决办法是使用第三方包selenium进行网页的自动测试。</p>
<p>安装使用selenium请参考<a href="https://blog.csdn.net/qq_42543254/article/details/81290257" target="_blank" rel="noopener">安装selenium及安装谷歌插件</a></p>
<figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">from</span> selenium <span class="keyword">import</span> webdriver</span><br><span class="line"><span class="keyword">import</span> re</span><br><span class="line"><span class="keyword">from</span> lxml <span class="keyword">import</span> etree</span><br><span class="line"><span class="keyword">from</span> selenium.webdriver.common.action_chains <span class="keyword">import</span> ActionChains</span><br><span class="line"><span class="keyword">import</span> time</span><br><span class="line"><span class="keyword">import</span> csv</span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">GetTiebaInfo</span><span class="params">(object)</span>:</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(self,url)</span>:</span></span><br><span class="line">        self.url = url</span><br><span class="line">        self.urls = self.getUrls(self.url)</span><br><span class="line">        self.items = self.spider(self.urls)</span><br><span class="line">        self.pipelines(self.items)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">getUrls</span><span class="params">(self,head_url)</span>:</span></span><br><span class="line">        urls = []</span><br><span class="line">        <span class="keyword">for</span> i <span class="keyword">in</span> range(<span class="number">97</span>,<span class="number">123</span>):</span><br><span class="line">            <span class="keyword">if</span> i == <span class="number">120</span>:</span><br><span class="line">                <span class="keyword">pass</span></span><br><span class="line">            <span class="keyword">else</span>:</span><br><span class="line">                url = head_url</span><br><span class="line">                url += chr(i)+<span class="string">'-index.html'</span></span><br><span class="line">                urls.append(url)</span><br><span class="line">        <span class="keyword">return</span> urls</span><br><span class="line"></span><br><span class="line">    <span class="comment">#爬虫模块</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">spider</span><span class="params">(self, urls)</span>:</span></span><br><span class="line">        items = []</span><br><span class="line">        <span class="keyword">for</span> url <span class="keyword">in</span> urls:</span><br><span class="line">            print(url)</span><br><span class="line">            browser = webdriver.Chrome()</span><br><span class="line">            browser.maximize_window()</span><br><span class="line">            browser.get(url)</span><br><span class="line"><span class="comment">#             left_click = browser.find_element_by_css_selector(".ranker.poptip.sort_default_asc.center")#按钮处理</span></span><br><span class="line"><span class="comment">#             ActionChains(browser).double_click(left_click).perform()</span></span><br><span class="line">            tree = etree.HTML(browser.page_source)</span><br><span class="line">            trTags = tree.xpath(<span class="string">'//tr[@data-row and not(@class)]'</span>)</span><br><span class="line">            <span class="keyword">for</span> tag <span class="keyword">in</span> trTags:</span><br><span class="line">                <span class="keyword">try</span>:</span><br><span class="line">                    item = []</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./th[1]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[1]/a/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[2]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[3]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[4]/a/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[5]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[6]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[7]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[8]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[9]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[10]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[11]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[12]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[13]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line">                    item.append(tag.xpath(<span class="string">'./td[14]/text()'</span>)[<span class="number">0</span>])</span><br><span class="line"><span class="comment">#                     if tag.xpath('./td[15]/text()'):</span></span><br><span class="line"><span class="comment">#                         item.append(tag.xpath('./td[15]/text()')[0])</span></span><br><span class="line"><span class="comment">#                     else:</span></span><br><span class="line"><span class="comment">#                         item.append('')</span></span><br><span class="line">                    items.append(item)</span><br><span class="line">                    item = []</span><br><span class="line">                <span class="keyword">except</span>:</span><br><span class="line">                    <span class="keyword">pass</span></span><br><span class="line">        <span class="keyword">return</span> items</span><br><span class="line"></span><br><span class="line">    <span class="comment">#对已经爬取的数据做后续处理</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">pipelines</span><span class="params">(self, items)</span>:</span></span><br><span class="line">        file_name = <span class="string">'basketball.csv'</span></span><br><span class="line">        <span class="keyword">with</span> open(file_name, <span class="string">'a'</span>, errors=<span class="string">'ignore'</span>, newline=<span class="string">''</span>,encoding=<span class="string">'utf-8'</span>) <span class="keyword">as</span> f:</span><br><span class="line">            f_csv = csv.writer(f)</span><br><span class="line">            f_csv.writerows(items)</span><br><span class="line">            print(<span class="string">'写入完毕'</span>)</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">'__main__'</span>:</span><br><span class="line">    url = <span class="string">u'https://www.sports-reference.com/cbb/coaches/'</span></span><br><span class="line">    GTI = GetTiebaInfo(url)</span><br></pre></td></tr></table></figure>

                

                <hr>
                <!-- Pager -->
                <ul class="pager">
                    
                        <li class="previous">
                            <a href="/article/spring-springmvc/" data-toggle="tooltip" data-placement="top" title="SSM框架学习（三）————SpringMVC">&larr; Previous Post</a>
                        </li>
                    
                    
                        <li class="next">
                            <a href="/article/python文本替换/" data-toggle="tooltip" data-placement="top" title="使用python进行文本替换（包括替换文件名、文件夹名、文本名）">Next Post &rarr;</a>
                        </li>
                    
                </ul>

                <br>

                <!--打赏-->
                
                <!--打赏-->

                <br>
                <!--分享-->
                
                    <div class="social-share"  data-wechat-qrcode-helper="" align="center"></div>
                    <!--  css & js -->
                    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/social-share.js/1.0.16/css/share.min.css">
                    <script src="https://cdnjs.cloudflare.com/ajax/libs/social-share.js/1.0.16/js/social-share.min.js"></script>
                
                <!--分享-->
                <br>                       
                
                <!-- require APlayer -->
                

                <!-- duoshuo Share start -->
                
                <!-- 多说 Share end-->

                <!-- 多说评论框 start -->
                
                <!-- 多说评论框 end -->

                <!-- disqus comment start -->
                
                <!-- disqus comment end -->

                

            </div>
            
            <!-- Tabe of Content -->
            <!-- Table of Contents -->

  
    <style>
      span.toc-nav-number{
        display: none
      }
    </style>
  
    
      <aside id="sidebar">
        <div id="toc" class="toc-article">
        <strong class="toc-title">Contents</strong>
        
          <ol class="toc-nav"><li class="toc-nav-item toc-nav-level-2"><a class="toc-nav-link" href="#一-观察url及要爬取的数据"><span class="toc-nav-number">1.</span> <span class="toc-nav-text">&#x4E00;&#x3001;&#x89C2;&#x5BDF;url&#x53CA;&#x8981;&#x722C;&#x53D6;&#x7684;&#x6570;&#x636E;</span></a></li><li class="toc-nav-item toc-nav-level-2"><a class="toc-nav-link" href="#二-对网页结构进行分析"><span class="toc-nav-number">2.</span> <span class="toc-nav-text">&#x4E8C;&#x3001;&#x5BF9;&#x7F51;&#x9875;&#x7ED3;&#x6784;&#x8FDB;&#x884C;&#x5206;&#x6790;</span></a></li><li class="toc-nav-item toc-nav-level-2"><a class="toc-nav-link" href="#三-数据爬取"><span class="toc-nav-number">3.</span> <span class="toc-nav-text">&#x4E09;&#x3001;&#x6570;&#x636E;&#x722C;&#x53D6;</span></a></li></ol>
        
        </div>
      </aside>
    

                
            <!-- Sidebar Container -->
            <div class="
                col-lg-8 col-lg-offset-2
                col-md-10 col-md-offset-1
                sidebar-container">

                <!-- Featured Tags -->
                
                <section>
                    <!-- no hr -->
                    <h5><a href="/tags/">FEATURED TAGS</a></h5>
                    <div class="tags">
                       
                          <a class="tag" href="/tags/#python" title="python">python</a>
                        
                          <a class="tag" href="/tags/#xpath" title="xpath">xpath</a>
                        
                          <a class="tag" href="/tags/#selenium" title="selenium">selenium</a>
                        
                          <a class="tag" href="/tags/#爬虫" title="爬虫">爬虫</a>
                        
                    </div>
                </section>
                

                <!-- Friends Blog -->
                
                <hr>
                <h5>FRIENDS</h5>
                <ul class="list-inline">

                    
                        <li><a href="https://blog.csdn.net/qq_40705355" target="_blank">CSDN Blog 赵先生</a></li>
                    
                        <li><a href="https://github.com/zqqpluto" target="_blank">Github 赵先生</a></li>
                    
                </ul>
                
            </div>
        </div>
    </div>
</article>








<!-- async load function -->
<script>
    function async(u, c) {
      var d = document, t = 'script',
          o = d.createElement(t),
          s = d.getElementsByTagName(t)[0];
      o.src = u;
      if (c) { o.addEventListener('load', function (e) { c(null, e); }, false); }
      s.parentNode.insertBefore(o, s);
    }
</script>
<!-- anchor-js, Doc:http://bryanbraun.github.io/anchorjs/ -->
<script>
    async("https://cdn.bootcss.com/anchor-js/1.1.1/anchor.min.js",function(){
        anchors.options = {
          visible: 'hover',
          placement: 'left',
          icon: 'ℬ'
        };
        anchors.add().remove('.intro-header h1').remove('.subheading').remove('.sidebar-container h5');
    })
</script>
<style>
    /* place left on bigger screen */
    @media all and (min-width: 800px) {
        .anchorjs-link{
            position: absolute;
            left: -0.75em;
            font-size: 1.1em;
            margin-top : -0.1em;
        }
    }
</style>


<!-- chrome Firefox 中文锚点定位失效-->
<script src="https://cdn.bootcss.com/jquery/3.3.1/jquery.js"></script>
<!-- smooth scroll behavior polyfill  -->
<script type="text/javascript" src="/js/smoothscroll.js"></script>
<script>
        $('#toc').on('click','a',function(a){
            // var isChrome = window.navigator.userAgent.indexOf("Chrome") !== -1;
            // console.log(window.navigator.userAgent,isChrome)
                // if(isChrome) {
                    // console.log(a.currentTarget.outerHTML);
                    // console.log($(a.currentTarget).attr("href"));
                    //跳转到指定锚点
                    // document.getElementById(a.target.innerText.toLowerCase()).scrollIntoView(true);
                    document.getElementById($(a.currentTarget).attr("href").replace("#","")).scrollIntoView({behavior: 'smooth' });
                // }
        })  
</script>


    <!-- Footer -->
    <!-- Footer -->
<footer>
    <div class="container">
        <div class="row">


            <div class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">



                
                <script src="//cdn1.lncld.net/static/js/3.0.4/av-min.js"></script>
                <!--Valine 的核心代码库:-->
                <script src="../../js/Valine.min.js"></script>
				<script src="https://cdnjs.loli.net/ajax/libs/jquery/3.2.1/jquery.min.js"></script>
                <div class="comment"></div>
                <script>
                    new Valine({
                        // AV 对象来自上面引入av-min.js(老司机们不要开车➳♡゛扎心了老铁)
                        av: AV,
                        el: '.comment', // 
						emoticon_url: 'https://cloud.panjunwen.com/alu',
                        notify: true, // 邮件提醒!!!
                        verify: false, // 验证码
                        app_id: 'aPeCQXmvnRjGpLqsHsTQ38nT-gzGzoHsz', // 这里填写上面得到的APP ID
                        app_key: 'ou0HF5y2Gu9oPP28sFN3ndGt', // 这里填写上面得到的APP KEY
                        placeholder: '欢迎留言评论' // [v1.0.7 new]留言框占位提示文字
						
                    });
					
                </script>
                



                <ul class="list-inline text-center">
                    
                    
                    

                    

                    

                    
                    <li>
                        <a target="_blank" href="https://github.com/zqqpluto">
                            <span class="fa-stack fa-lg">
                                <i class="fa fa-circle fa-stack-2x"></i>
                                <i class="fa fa-github fa-stack-1x fa-inverse"></i>
                            </span>
                        </a>
                    </li>
                    

                    

                </ul>


                <p class="copyright text-muted">

                    
                    <!-- 不蒜子统计 -->
                    <span id="busuanzi_container_site_uv" style='display:none'>
                        本站访客数<span id="busuanzi_value_site_uv"></span>人
                    </span>
                    <script async src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script>
                    

                    Copyright &copy; 赵先生 2019
                    <br>
                    Theme by <a href="http://beantech.org">BeanTech</a>
                    <span style="display: inline-block; margin: 0 5px;">
                        <i class="fa fa-heart"></i>
                    </span>
                    re-Ported by <a href="https://huweihuang.github.io/">赵先生</a> |
                    <iframe style="margin-left: 2px; margin-bottom:-5px;" frameborder="0" scrolling="0" width="91px"
                        height="20px"
                        src="https://ghbtns.com/github-btn.html?user=huweihuang&repo=hexo-theme-huweihuang&type=star&count=true">
                    </iframe>
                </p>
            </div>
        </div>
    </div>
</footer>

<!-- jQuery -->
<script src="/js/jquery.min.js"></script>

<!-- Bootstrap Core JavaScript -->
<script src="/js/bootstrap.min.js"></script>

<!-- Custom Theme JavaScript -->
<script src="/js/hux-blog.min.js"></script>


<!-- async load function -->
<script>
    function async(u, c) {
        var d = document, t = 'script',
            o = d.createElement(t),
            s = d.getElementsByTagName(t)[0];
        o.src = u;
        if (c) { o.addEventListener('load', function (e) { c(null, e); }, false); }
        s.parentNode.insertBefore(o, s);
    }
</script>

<!-- 
     Because of the native support for backtick-style fenced code blocks 
     right within the Markdown is landed in Github Pages, 
     From V1.6, There is no need for Highlight.js, 
     so Huxblog drops it officially.

     - https://github.com/blog/2100-github-pages-now-faster-and-simpler-with-jekyll-3-0  
     - https://help.github.com/articles/creating-and-highlighting-code-blocks/    
-->
<!--
    <script>
        async("http://cdn.bootcss.com/highlight.js/8.6/highlight.min.js", function(){
            hljs.initHighlightingOnLoad();
        })
    </script>
    <link href="http://cdn.bootcss.com/highlight.js/8.6/styles/github.min.css" rel="stylesheet">
-->


<!-- jquery.tagcloud.js -->
<script>

    // only load tagcloud.js in tag.html
    if ($('#tag_cloud').length !== 0) {
        async("http://zqqpluto.xyz/js/jquery.tagcloud.js", function () {
            $.fn.tagcloud.defaults = {
                //size: {start: 1, end: 1, unit: 'em'},
                color: { start: '#bbbbee', end: '#0085a1' },
            };
            $('#tag_cloud a').tagcloud();
        })
    }
</script>


<!-- <script>

//----自定义js----------------
function createImgEventFullScreen() {
  var imgs = $(".post-container").find("img");
  console.log(imgs);
  for(var i = 0;i < imgs.length;i++) {
    // $(imgs[i]).click(createCover(imgs[i]));
    imgs[i].onclick= function(e) {
      var src = e.srcElement.currentSrc;
      createCover(src)
    }
  }
  function createCover (src) {
    console.log(src);
    var cover = $("<div id='fullScreenCover' class='zhao-cover-img-container'><img class='zhao-cover-img' src='"+src+"'/></div>");
    $("#fullScreenCover").remove();
    $("body").append(cover);
    $("body").addClass("zhao-no-scroll");
    $("#fullScreenCover").click(function(){
      $("#fullScreenCover").remove();
      $("body").removeClass("zhao-no-scroll");
    })
  }
}
setTimeout(function(){
  createImgEventFullScreen();
},1000)


</script> -->


<!--fastClick.js -->
<script>
    async("https://cdn.bootcss.com/fastclick/1.0.6/fastclick.min.js", function () {
        var $nav = document.querySelector("nav");
        if ($nav) FastClick.attach($nav);
    })
</script>


<!-- Google Analytics -->


<script>
    // dynamic User by Hux
    var _gaId = 'UA-XXXXXXXX-X';
    var _gaDomain = 'yoursite';

    // Originial
    (function (i, s, o, g, r, a, m) {
        i['GoogleAnalyticsObject'] = r; i[r] = i[r] || function () {
            (i[r].q = i[r].q || []).push(arguments)
        }, i[r].l = 1 * new Date(); a = s.createElement(o),
            m = s.getElementsByTagName(o)[0]; a.async = 1; a.src = g; m.parentNode.insertBefore(a, m)
    })(window, document, 'script', '//www.google-analytics.com/analytics.js', 'ga');

    ga('create', _gaId, _gaDomain);
    ga('send', 'pageview');
</script>




<!-- Baidu Tongji -->

<script>
    // dynamic User by Hux
    var _baId = 'xxx';

    // Originial
    var _hmt = _hmt || [];
    (function () {
        var hm = document.createElement("script");
        hm.src = "//hm.baidu.com/hm.js?" + _baId;
        var s = document.getElementsByTagName("script")[0];
        s.parentNode.insertBefore(hm, s);
    })();
</script>



<script>
    function createImgEventFullScreen() {
        var imgs = $(".post-container").find("img");
        console.log(imgs);
        for (var i = 0; i < imgs.length; i++) {
            // $(imgs[i]).click(createCover(imgs[i]));
            imgs[i].onclick = function (e) {
                var src = e.srcElement.currentSrc;
                var _this = $(this);
                console.log(_this);
                createCover(src,_this);
            }
        }
        function createCover(src,_this) {
            console.log(_this);
            console.log(src);
            var cover = $("<div id='outerDiv'  style='position:fixed;top:0;left:0;background:rgba(0,0,0,0.7);z-index:5;width:100%;height:100%;display:none;'><div id='innerDiv' style='position:absolute;'><img  id='bigImg' style='border:5px solid #fff;' src=''/></div></div>");
            $("#outerDiv").remove();
            $("body").append(cover);
            imgShow("#outerDiv", "#innerDiv", "#bigImg", _this,src);

        }
    }
    function imgShow(outerDiv, innerDiv, bigImg, _this,src) {
        //var src = _this.attr("src"); //获取当前点击的common-img元素中的src属性
        $(bigImg).attr("src", src); //设置#bigImg元素的src属性

        /*获取当前点击图片的真实大小，并显示弹出层及大图*/
        $("<img/>").attr("src", src).load(function () {
            var windowW = $(window).width(); //获取当前窗口宽度
            var windowH = $(window).height(); //获取当前窗口高度
            var realWidth = this.width; //获取图片真实宽度
            var realHeight = this.height; //获取图片真实高度
            var imgWidth, imgHeight;
            var scale = 0.8; //缩放尺寸，当图片真实宽度和高度大于窗口宽度和高度时进行缩放

            if (realHeight > windowH * scale) { //判断图片高度
                imgHeight = windowH * scale; //如大于窗口高度，图片高度进行缩放
                imgWidth = imgHeight / realHeight * realWidth; //等比例缩放宽度
                if (imgWidth > windowW * scale) { //如宽度仍大于窗口宽度
                    imgWidth = windowW * scale; //再对宽度进行缩放
                }
            } else if (realWidth > windowW * scale) { //如图片高度合适，判断图片宽度
                imgWidth = windowW * scale; //如大于窗口宽度，图片宽度进行缩放
                imgHeight = imgWidth / realWidth * realHeight; //等比例缩放高度
            } else { //如果图片真实高度和宽度都符合要求，高宽不变
                imgWidth = realWidth;
                imgHeight = realHeight;
            }
            $(bigImg).css("width", imgWidth); //以最终的宽度对图片缩放

            var w = (windowW - imgWidth) / 2; //计算图片与窗口左边距
            var h = (windowH - imgHeight) / 2; //计算图片与窗口上边距
            $(innerDiv).css({ "top": h, "left": w }); //设置#innerDiv的top和left属性
            //console.log('****')
            $(outerDiv).fadeIn("fast"); //淡入显示#outerDiv
        });

        $(outerDiv).click(function () { //再次点击淡出消失弹出层
            $(this).fadeOut("fast");
        });
    }
    setTimeout(function () {
        createImgEventFullScreen();
    }, 1000)


</script>


	<a id="rocket" href="#top" class=""></a>
	<script type="text/javascript" src="/js/totop.js?v=1.0.0" async=""></script>
    <script type="text/javascript" src="/js/toc.js?v=1.0.0" async=""></script>
<!-- Image to hack wechat -->
<img src="http://zqqpluto.xyz/img/icon_wechat.png" width="0" height="0" />
<!-- Migrate from head to bottom, no longer block render and still work -->

<!-- 代码块复制功能 -->
<script type="text/javascript" src="/js/clipboard.min.js"></script>  
<script type="text/javascript" src="/js/clipboard-use.js"></script>
</body>

</html>
